{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import numpy as np\n",
    "import pickle\n",
    "import multiprocessing\n",
    "from gensim.models import Word2Vec\n",
    "from gensim.models.word2vec import LineSentence\n",
    "import pandas as pd\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import jieba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "data = pd.read_csv('../data/atec_nlp_sim_train.csv',sep='\\t',encoding='utf8', names=['id', 'q1', 'q2', 'label'] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>q1</th>\n",
       "      <th>q2</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>﻿怎么更改花呗手机号码</td>\n",
       "      <td>我的花呗是以前的手机号码，怎么更改成现在的支付宝的号码手机号</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>也开不了花呗，就这样了？完事了</td>\n",
       "      <td>真的嘛？就是花呗付款</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>花呗冻结以后还能开通吗</td>\n",
       "      <td>我的条件可以开通花呗借款吗</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>如何得知关闭借呗</td>\n",
       "      <td>想永久关闭借呗</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>花呗扫码付钱</td>\n",
       "      <td>二维码扫描可以用花呗吗</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id               q1                              q2  label\n",
       "0   1      ﻿怎么更改花呗手机号码  我的花呗是以前的手机号码，怎么更改成现在的支付宝的号码手机号      1\n",
       "1   2  也开不了花呗，就这样了？完事了                      真的嘛？就是花呗付款      0\n",
       "2   3      花呗冻结以后还能开通吗                   我的条件可以开通花呗借款吗      0\n",
       "3   4         如何得知关闭借呗                         想永久关闭借呗      0\n",
       "4   5           花呗扫码付钱                     二维码扫描可以用花呗吗      0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    83792\n",
       "1    18685\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.label.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "83792/18685"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache /tmp/jieba.cache\n",
      "Loading model cost 7.165 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    }
   ],
   "source": [
    "data['q1_cut']=data['q1'].map(lambda x:list(jieba.cut(x)))\n",
    "data['q2_cut']=data['q2'].map(lambda x:list(jieba.cut(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data.to_csv('../data/cut.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "content =list(data.q1_cut)+list(data.q2_cut)\n",
    "\n",
    "\n",
    "model = Word2Vec(content, size=config['w2v_vec_dim'], window=5, min_count=5, workers=multiprocessing.cpu_count()\n",
    "                 )\n",
    "model.save(config['w2v_content_word_model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2552, 256)\n",
      "save\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bigdata/anaconda3/envs/py2/lib/python2.7/site-packages/ipykernel/__main__.py:1: DeprecationWarning: Call to deprecated `syn0` (Attribute will be removed in 4.0.0, use self.wv.vectors instead).\n",
      "  if __name__ == '__main__':\n"
     ]
    }
   ],
   "source": [
    "weights = model.wv.syn0\n",
    "vocab = dict([(k, v.index + 1) for k, v in model.wv.vocab.items()])\n",
    "vocab['<-UNKNOW->'] = len(vocab) + 1\n",
    "embed_weights = np.zeros(shape=(weights.shape[0] + 2, weights.shape[1]))\n",
    "embed_weights[1:weights.shape[0] + 1] = weights\n",
    "unk_vec = np.random.random(size=weights.shape[1]) * 0.5\n",
    "pading_vec = np.random.random(size=weights.shape[1]) * 0\n",
    "embed_weights[weights.shape[0] + 1] = unk_vec - unk_vec.mean()\n",
    "embed_weights[0] = pading_vec\n",
    "\n",
    "#pickle.dump(vocab, open(config['word_embed_dict'], \"wb\"))\n",
    "\n",
    "config['vocab_size'] = embed_weights.shape[0]\n",
    "config['w2v_vec_dim'] = embed_weights.shape[1]\n",
    "yaml.dump(config, open('config.yaml', 'w'))\n",
    "\n",
    "np.save(config['word_embed_weight'], embed_weights)\n",
    "print(embed_weights.shape)\n",
    "print('save')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word_voc = vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "def padding_id(ids, padding_token=0, padding_length=None):\n",
    "    if len(ids) > padding_length:\n",
    "        return ids[:padding_length]\n",
    "    else:\n",
    "        return ids + [padding_token] * (padding_length - len(ids))\n",
    "\n",
    "\n",
    "def word2id(contents):\n",
    "    ''' contents  list\n",
    "    '''\n",
    "#     contents = str(contents)\n",
    "#     contents = contents.split()\n",
    "\n",
    "    ids = [word_voc[c] if c in word_voc else len(word_voc) for c in contents]\n",
    "\n",
    "    return padding_id(ids, padding_token=0, padding_length=config['word_maxlen'])\n",
    "\n",
    "\n",
    "def data2id():\n",
    "\n",
    "    data = read_df(config['train_cut'])\n",
    "    data['title'] = data.title.map(lambda x: word2id(x))\n",
    "    data['content'] = data.content.map(lambda x: word2id(x))\n",
    "    print(data.head(4))\n",
    "    data.to_pickle(config['train_cut_id_dump'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data['q1_cut_id'] = data['q1_cut'].map(lambda x: word2id(x))\n",
    "data['q2_cut_id'] = data['q2_cut'].map(lambda x: word2id(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>q1</th>\n",
       "      <th>q2</th>\n",
       "      <th>label</th>\n",
       "      <th>q1_cut</th>\n",
       "      <th>q2_cut</th>\n",
       "      <th>q1_cut_id</th>\n",
       "      <th>q2_cut_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>﻿1</td>\n",
       "      <td>﻿怎么更改花呗手机号码</td>\n",
       "      <td>我的花呗是以前的手机号码，怎么更改成现在的支付宝的号码手机号</td>\n",
       "      <td>1</td>\n",
       "      <td>[﻿, 怎么, 更改, 花, 呗, 手机号码]</td>\n",
       "      <td>[我, 的, 花, 呗, 是, 以前, 的, 手机号码, ，, 怎么, 更, 改成, 现在,...</td>\n",
       "      <td>[2551, 10, 233, 2, 1, 216, 0, 0, 0, 0, 0, 0, 0...</td>\n",
       "      <td>[5, 4, 2, 1, 19, 159, 4, 216, 8, 10, 986, 603,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>也开不了花呗，就这样了？完事了</td>\n",
       "      <td>真的嘛？就是花呗付款</td>\n",
       "      <td>0</td>\n",
       "      <td>[也, 开, 不了, 花, 呗, ，, 就, 这样, 了, ？, 完事, 了]</td>\n",
       "      <td>[真的, 嘛, ？, 就是, 花, 呗, 付款]</td>\n",
       "      <td>[103, 160, 32, 2, 1, 8, 74, 623, 7, 139, 2479,...</td>\n",
       "      <td>[802, 131, 139, 225, 2, 1, 35, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>花呗冻结以后还能开通吗</td>\n",
       "      <td>我的条件可以开通花呗借款吗</td>\n",
       "      <td>0</td>\n",
       "      <td>[花, 呗, 冻结, 以后, 还, 能, 开通, 吗]</td>\n",
       "      <td>[我, 的, 条件, 可以, 开通, 花, 呗, 借款, 吗]</td>\n",
       "      <td>[2, 1, 110, 178, 11, 23, 20, 9, 0, 0, 0, 0, 0,...</td>\n",
       "      <td>[5, 4, 186, 14, 20, 2, 1, 120, 9, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>如何得知关闭借呗</td>\n",
       "      <td>想永久关闭借呗</td>\n",
       "      <td>0</td>\n",
       "      <td>[如何, 得知, 关闭, 借, 呗]</td>\n",
       "      <td>[想, 永久, 关闭, 借, 呗]</td>\n",
       "      <td>[55, 2551, 53, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0...</td>\n",
       "      <td>[70, 574, 53, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>花呗扫码付钱</td>\n",
       "      <td>二维码扫描可以用花呗吗</td>\n",
       "      <td>0</td>\n",
       "      <td>[花, 呗, 扫码, 付钱]</td>\n",
       "      <td>[二维码, 扫描, 可以, 用花, 呗, 吗]</td>\n",
       "      <td>[2, 1, 311, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n",
       "      <td>[210, 868, 14, 33, 1, 9, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id               q1                              q2  label  \\\n",
       "0  ﻿1      ﻿怎么更改花呗手机号码  我的花呗是以前的手机号码，怎么更改成现在的支付宝的号码手机号      1   \n",
       "1   2  也开不了花呗，就这样了？完事了                      真的嘛？就是花呗付款      0   \n",
       "2   3      花呗冻结以后还能开通吗                   我的条件可以开通花呗借款吗      0   \n",
       "3   4         如何得知关闭借呗                         想永久关闭借呗      0   \n",
       "4   5           花呗扫码付钱                     二维码扫描可以用花呗吗      0   \n",
       "\n",
       "                                    q1_cut  \\\n",
       "0                  [﻿, 怎么, 更改, 花, 呗, 手机号码]   \n",
       "1  [也, 开, 不了, 花, 呗, ，, 就, 这样, 了, ？, 完事, 了]   \n",
       "2              [花, 呗, 冻结, 以后, 还, 能, 开通, 吗]   \n",
       "3                       [如何, 得知, 关闭, 借, 呗]   \n",
       "4                           [花, 呗, 扫码, 付钱]   \n",
       "\n",
       "                                              q2_cut  \\\n",
       "0  [我, 的, 花, 呗, 是, 以前, 的, 手机号码, ，, 怎么, 更, 改成, 现在,...   \n",
       "1                           [真的, 嘛, ？, 就是, 花, 呗, 付款]   \n",
       "2                    [我, 的, 条件, 可以, 开通, 花, 呗, 借款, 吗]   \n",
       "3                                  [想, 永久, 关闭, 借, 呗]   \n",
       "4                            [二维码, 扫描, 可以, 用花, 呗, 吗]   \n",
       "\n",
       "                                           q1_cut_id  \\\n",
       "0  [2551, 10, 233, 2, 1, 216, 0, 0, 0, 0, 0, 0, 0...   \n",
       "1  [103, 160, 32, 2, 1, 8, 74, 623, 7, 139, 2479,...   \n",
       "2  [2, 1, 110, 178, 11, 23, 20, 9, 0, 0, 0, 0, 0,...   \n",
       "3  [55, 2551, 53, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0...   \n",
       "4  [2, 1, 311, 556, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...   \n",
       "\n",
       "                                           q2_cut_id  \n",
       "0  [5, 4, 2, 1, 19, 159, 4, 216, 8, 10, 986, 603,...  \n",
       "1  [802, 131, 139, 225, 2, 1, 35, 0, 0, 0, 0, 0, ...  \n",
       "2  [5, 4, 186, 14, 20, 2, 1, 120, 9, 0, 0, 0, 0, ...  \n",
       "3  [70, 574, 53, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,...  \n",
       "4  [210, 868, 14, 33, 1, 9, 0, 0, 0, 0, 0, 0, 0, ...  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras.models import *\n",
    "from keras.layers import *\n",
    "from keras.optimizers import *\n",
    "from keras.preprocessing import sequence\n",
    "from keras.regularizers import l2\n",
    "from keras import backend as K\n",
    "from keras.engine.topology import Layer\n",
    "from keras.backend.tensorflow_backend import set_session\n",
    "import time\n",
    "from tensorflow.contrib import learn\n",
    "import yaml\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import precision_score, recall_score, f1_score\n",
    "import numpy as np\n",
    "from keras import backend as K\n",
    "\n",
    "params = yaml.load(open('params.yaml', 'r'))\n",
    "config = yaml.load(open('config.yaml', 'r'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def convs_block(data, convs=[3,4,5], f=256, name=\"conv_feat\"):\n",
    "    pools = []\n",
    "    for c in convs:\n",
    "        conv = Activation(activation=\"relu\")(BatchNormalization()(\n",
    "            Conv1D(filters=f, kernel_size=c, padding=\"valid\")(data)))\n",
    "        pool = GlobalMaxPool1D()(conv)\n",
    "        pools.append(pool)\n",
    "    return concatenate(pools, name=name)\n",
    "\n",
    "\n",
    "def convs_block2(data, convs=[3, 4, 5], f=256, name=\"conv_feat\"):\n",
    "    pools = []\n",
    "    for c in convs:\n",
    "        conv = Activation(activation=\"relu\")(BatchNormalization()(\n",
    "            Conv1D(filters=f, kernel_size=c, padding=\"valid\")(data)))\n",
    "        conv = MaxPool1D(pool_size=10)(conv)\n",
    "        conv = Activation(activation=\"relu\")(BatchNormalization()(\n",
    "            Conv1D(filters=f, kernel_size=c, padding=\"valid\")(conv)))\n",
    "\n",
    "        pool = GlobalMaxPool1D()(conv)\n",
    "        pools.append(pool)\n",
    "    return concatenate(pools, name=name)\n",
    "\n",
    "\n",
    "def get_textcnn(seq_length, embed_weight, pretrain=False):\n",
    "    content = Input(shape=(seq_length,), dtype=\"int32\")\n",
    "    if pretrain:\n",
    "        embedding = Embedding(name='word_embedding', input_dim=config[\n",
    "                              'vocab_size'], weights=[embed_weight], output_dim=config['w2v_vec_dim'], trainable=False)\n",
    "    else:\n",
    "        embedding = Embedding(name='word_embedding', input_dim=config[\n",
    "                              'vocab_size'], output_dim=config['w2v_vec_dim'], trainable=True)\n",
    "    trans_content = Activation(activation=\"relu\")(\n",
    "        BatchNormalization()((TimeDistributed(Dense(256))(embedding(content)))))\n",
    "    feat = convs_block(trans_content)\n",
    "    dropfeat = Dropout(0.2)(feat)\n",
    "    fc = Activation(activation=\"relu\")(\n",
    "        BatchNormalization()(Dense(256)(dropfeat)))\n",
    "    output = Dense(2, activation=\"softmax\")(fc)\n",
    "    model = Model(inputs=content, outputs=output)\n",
    "    model.compile(loss='categorical_crossentropy',\n",
    "                  optimizer=\"adam\", metrics=['accuracy'])\n",
    "    model.summary()\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import keras\n",
    "from keras.models import *\n",
    "from keras.layers import *\n",
    "from keras.optimizers import *\n",
    "from keras.preprocessing import sequence\n",
    "from keras.regularizers import l2\n",
    "from keras import backend as K\n",
    "from keras.engine.topology import Layer\n",
    "from keras.backend.tensorflow_backend import set_session\n",
    "#from recurrentshop import *\n",
    "import time\n",
    "from tensorflow.contrib import learn\n",
    "import yaml\n",
    "from sklearn.model_selection import train_test_split\n",
    "import keras.backend as K\n",
    "from keras.callbacks import TensorBoard\n",
    "params = yaml.load(open('params.yaml', 'r'))\n",
    "config = yaml.load(open('config.yaml', 'r'))\n",
    "\n",
    "from keras.callbacks import Callback\n",
    "from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, precision_score, recall_score\n",
    "\n",
    "\n",
    "\n",
    "def score(label, pred, gate=0.5):\n",
    "\n",
    "    if len(label.shape) == 1:\n",
    "        p = (pred > gate).astype(\"int\")\n",
    "        p = np.squeeze(p)\n",
    "        l = label\n",
    "    else:\n",
    "        p = np.argmax(pred, axis=1)\n",
    "        l = np.argmax(label, axis=1)\n",
    "\n",
    "    #print(confusion_matrix(l, p).view())\n",
    "    pre_score = precision_score(l, p, pos_label=1, average='binary')\n",
    "    rec_score = recall_score(l, p, pos_label=1, average='binary')\n",
    "    f_score = f1_score(l, p)\n",
    "    return pre_score, rec_score, f_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "q1_id = np.array(list(data.q1_cut_id.values))\n",
    "q2_id = np.array(list(data.q2_cut_id.values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(39346, 20)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q1_id.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(39346, 40)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.concatenate([q1_id,q2_id],1).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data['label'] = data['label'].apply(\n",
    "        lambda x: int(x ==1))  # 人写的是1 机器写的是0\n",
    "X = np.concatenate([q1_id,q2_id],1)\n",
    "Y = keras.utils.to_categorical(list(data.label), num_classes=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "x_train, x_dev, y_train, y_dev = train_test_split(X, Y,    test_size=0.1,     random_state=2008)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_batch_generator(x_source, y_source, batch):\n",
    "    while True:\n",
    "        batch_list_x = []\n",
    "        batch_list_y = []\n",
    "        for line, y in zip(x_source, y_source):\n",
    "            x = line.astype('float32')\n",
    "            batch_list_x.append(x)\n",
    "            batch_list_y.append(y)\n",
    "            if len(batch_list_x) == batch:\n",
    "                yield (np.array(batch_list_x), np.array(batch_list_y))\n",
    "                batch_list_x = []\n",
    "                batch_list_y = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_4 (InputLayer)            (None, 40)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "word_embedding (Embedding)      (None, 40, 256)      653312      input_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_4 (TimeDistrib (None, 40, 256)      65792       word_embedding[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_20 (BatchNo (None, 40, 256)      1024        time_distributed_4[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "activation_20 (Activation)      (None, 40, 256)      0           batch_normalization_20[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_14 (Conv1D)              (None, 38, 256)      196864      activation_20[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_15 (Conv1D)              (None, 37, 256)      262400      activation_20[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_16 (Conv1D)              (None, 36, 256)      327936      activation_20[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_21 (BatchNo (None, 38, 256)      1024        conv1d_14[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_22 (BatchNo (None, 37, 256)      1024        conv1d_15[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_23 (BatchNo (None, 36, 256)      1024        conv1d_16[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_21 (Activation)      (None, 38, 256)      0           batch_normalization_21[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_22 (Activation)      (None, 37, 256)      0           batch_normalization_22[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "activation_23 (Activation)      (None, 36, 256)      0           batch_normalization_23[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_14 (Global (None, 256)          0           activation_21[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_15 (Global (None, 256)          0           activation_22[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_16 (Global (None, 256)          0           activation_23[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv_feat (Concatenate)         (None, 768)          0           global_max_pooling1d_14[0][0]    \n",
      "                                                                 global_max_pooling1d_15[0][0]    \n",
      "                                                                 global_max_pooling1d_16[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)             (None, 768)          0           conv_feat[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_11 (Dense)                (None, 256)          196864      dropout_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_24 (BatchNo (None, 256)          1024        dense_11[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_24 (Activation)      (None, 256)          0           batch_normalization_24[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_12 (Dense)                (None, 2)            514         activation_24[0][0]              \n",
      "==================================================================================================\n",
      "Total params: 1,708,802\n",
      "Trainable params: 1,706,242\n",
      "Non-trainable params: 2,560\n",
      "__________________________________________________________________________________________________\n",
      "Epoch 1/1\n",
      "276/276 [==============================] - 233s 843ms/step - loss: 0.5424 - acc: 0.7645 - val_loss: 0.6513 - val_acc: 0.7855\n",
      "('p r f1 ', 1.0, 0.007058823529411765, 0.014018691588785047)\n",
      "Epoch 1/1\n",
      "276/276 [==============================] - 229s 831ms/step - loss: 0.4759 - acc: 0.7880 - val_loss: 0.5375 - val_acc: 0.7896\n",
      "('p r f1 ', 0.59482758620689657, 0.081176470588235294, 0.14285714285714285)\n",
      "Epoch 1/1\n",
      "276/276 [==============================] - 229s 828ms/step - loss: 0.4397 - acc: 0.8035 - val_loss: 0.5999 - val_acc: 0.7098\n",
      "('p r f1 ', 0.34823284823284822, 0.39411764705882352, 0.36975717439293598)\n",
      "Epoch 1/1\n",
      "276/276 [==============================] - 229s 828ms/step - loss: 0.4016 - acc: 0.8194 - val_loss: 0.5772 - val_acc: 0.7166\n",
      "('p r f1 ', 0.35326688815060908, 0.37529411764705883, 0.36394751853964635)\n",
      "Epoch 1/1\n",
      "276/276 [==============================] - 227s 823ms/step - loss: 0.3589 - acc: 0.8407 - val_loss: 0.6428 - val_acc: 0.7271\n",
      "('p r f1 ', 0.36407766990291263, 0.35294117647058826, 0.35842293906810041)\n",
      "Epoch 1/1\n",
      "276/276 [==============================] - 227s 823ms/step - loss: 0.3178 - acc: 0.8600 - val_loss: 0.6930 - val_acc: 0.7685\n",
      "('p r f1 ', 0.41952506596306066, 0.18705882352941178, 0.25874694873881204)\n",
      "Epoch 1/1\n",
      "276/276 [==============================] - 228s 826ms/step - loss: 0.2777 - acc: 0.8802 - val_loss: 0.7691 - val_acc: 0.7588\n",
      "('p r f1 ', 0.39445628997867804, 0.21764705882352942, 0.28051554207733137)\n",
      "Epoch 1/1\n",
      "276/276 [==============================] - 231s 836ms/step - loss: 0.2460 - acc: 0.8957 - val_loss: 0.7811 - val_acc: 0.7543\n",
      "('p r f1 ', 0.39572192513368987, 0.26117647058823529, 0.31467044649184972)\n",
      "Epoch 1/1\n",
      " 32/276 [==>...........................] - ETA: 3:14 - loss: 0.2511 - acc: 0.8945"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-35-0916f334efbe>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     10\u001b[0m         \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m         \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_train\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'batch_size'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m         \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_dev\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_dev\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m     )\n",
      "\u001b[0;32m/home/bigdata/anaconda3/envs/py2/lib/python2.7/site-packages/keras/legacy/interfaces.pyc\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     89\u001b[0m                 warnings.warn('Update your `' + object_name +\n\u001b[1;32m     90\u001b[0m                               '` call to the Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 91\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     92\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/bigdata/anaconda3/envs/py2/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m   2222\u001b[0m                     outs = self.train_on_batch(x, y,\n\u001b[1;32m   2223\u001b[0m                                                \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2224\u001b[0;31m                                                class_weight=class_weight)\n\u001b[0m\u001b[1;32m   2225\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2226\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/bigdata/anaconda3/envs/py2/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, sample_weight, class_weight)\u001b[0m\n\u001b[1;32m   1881\u001b[0m             \u001b[0mins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1882\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1883\u001b[0;31m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1884\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1885\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/bigdata/anaconda3/envs/py2/lib/python2.7/site-packages/keras/backend/tensorflow_backend.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2476\u001b[0m         \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2477\u001b[0m         updated = session.run(fetches=fetches, feed_dict=feed_dict,\n\u001b[0;32m-> 2478\u001b[0;31m                               **self.session_kwargs)\n\u001b[0m\u001b[1;32m   2479\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2480\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/bigdata/anaconda3/envs/py2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    893\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    894\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 895\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    896\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    897\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/bigdata/anaconda3/envs/py2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1126\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1127\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1128\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1129\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1130\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/bigdata/anaconda3/envs/py2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1342\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1343\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1344\u001b[0;31m                            options, run_metadata)\n\u001b[0m\u001b[1;32m   1345\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1346\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/bigdata/anaconda3/envs/py2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1348\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1349\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1350\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1351\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1352\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/bigdata/anaconda3/envs/py2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1327\u001b[0m           return tf_session.TF_Run(session, options,\n\u001b[1;32m   1328\u001b[0m                                    \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1329\u001b[0;31m                                    status, run_metadata)\n\u001b[0m\u001b[1;32m   1330\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1331\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model = get_textcnn(config['word_maxlen']*2,\n",
    "                        embed_weights, pretrain=False)\n",
    "\n",
    "for i in range(15):\n",
    "    if i == 8:\n",
    "        K.set_value(model.optimizer.lr, 0.0001)\n",
    "\n",
    "    model.fit_generator(\n",
    "        train_batch_generator(x_train, y_train, config['batch_size']),\n",
    "        epochs=1,\n",
    "        steps_per_epoch=int(x_train.shape[0] / config['batch_size']),\n",
    "        validation_data=(x_dev, y_dev),\n",
    "    \n",
    "    )\n",
    "\n",
    "    pred = model.predict(x_dev, batch_size=config['batch_size'])\n",
    "    pre, rec, f1 = score(y_dev, pred)\n",
    "\n",
    "    print('p r f1 ', pre, rec, f1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_textrnn(seq_length, embed_weight, pretrain=False):\n",
    "    main_input = Input(shape=(seq_length,), dtype='float64')\n",
    "    if pretrain:\n",
    "        embedding = Embedding(name='word_embedding', input_dim=config[\n",
    "            'vocab_size'], weights=[embed_weight], output_dim=config['w2v_vec_dim'], trainable=False)\n",
    "    else:\n",
    "        embedding = Embedding(name='word_embedding', input_dim=config[\n",
    "            'vocab_size'], output_dim=config['w2v_vec_dim'], trainable=True)\n",
    "    out = embedding(main_input)\n",
    "    out = Activation(activation=\"relu\")(\n",
    "        BatchNormalization()((TimeDistributed(Dense(256))(out))))\n",
    "    print('Build model...')\n",
    "    out = Bidirectional(GRU(256))(out)\n",
    "    #out= BatchNormalization(out)\n",
    "    out = Dropout(0.3)(out)\n",
    "    fc = Activation(activation=\"relu\")(\n",
    "        BatchNormalization()(Dense(256)(out)))\n",
    "    main_output = Dense(config['number_classes'],\n",
    "                        activation='softmax')(fc)\n",
    "\n",
    "    model = Model(inputs=main_input, outputs=main_output)\n",
    "    model.compile(optimizer='adam',\n",
    "                  loss='categorical_crossentropy',\n",
    "                  metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Build model...\n",
      "Epoch 1/1\n",
      "276/276 [==============================] - 682s 2s/step - loss: 0.5495 - acc: 0.7704 - val_loss: 0.5360 - val_acc: 0.7629\n",
      "('p r f1 ', 0.34909090909090912, 0.11294117647058824, 0.17066666666666669)\n",
      "Epoch 1/1\n",
      "276/276 [==============================] - 671s 2s/step - loss: 0.4976 - acc: 0.7841 - val_loss: 0.5197 - val_acc: 0.7776\n",
      "('p r f1 ', 0.44680851063829785, 0.12352941176470589, 0.19354838709677422)\n",
      "Epoch 1/1\n",
      "276/276 [==============================] - 673s 2s/step - loss: 0.4629 - acc: 0.7953 - val_loss: 0.5173 - val_acc: 0.7751\n",
      "('p r f1 ', 0.44444444444444442, 0.16470588235294117, 0.24034334763948498)\n",
      "Epoch 1/1\n",
      "276/276 [==============================] - 653s 2s/step - loss: 0.4329 - acc: 0.8070 - val_loss: 0.5357 - val_acc: 0.7677\n",
      "('p r f1 ', 0.41160220994475138, 0.17529411764705882, 0.24587458745874588)\n",
      "Epoch 1/1\n",
      "276/276 [==============================] - 663s 2s/step - loss: 0.4004 - acc: 0.8234 - val_loss: 0.6243 - val_acc: 0.6574\n",
      "('p r f1 ', 0.28534482758620688, 0.38941176470588235, 0.32935323383084575)\n",
      "Epoch 1/1\n",
      "276/276 [==============================] - 1123s 4s/step - loss: 0.3674 - acc: 0.8361 - val_loss: 0.6200 - val_acc: 0.6864\n",
      "('p r f1 ', 0.29704016913319237, 0.33058823529411763, 0.31291759465478841)\n",
      "Epoch 1/1\n",
      "276/276 [==============================] - 1477s 5s/step - loss: 0.3410 - acc: 0.8485 - val_loss: 0.8609 - val_acc: 0.5372\n",
      "('p r f1 ', 0.25977238990598711, 0.61764705882352944, 0.36572622779519326)\n",
      "Epoch 1/1\n",
      "276/276 [==============================] - 1146s 4s/step - loss: 0.3017 - acc: 0.8668 - val_loss: 0.7011 - val_acc: 0.7266\n",
      "('p r f1 ', 0.33717579250720459, 0.2752941176470588, 0.30310880829015541)\n",
      "Epoch 1/1\n",
      " 78/276 [=======>......................] - ETA: 9:06 - loss: 0.3315 - acc: 0.8501"
     ]
    }
   ],
   "source": [
    "\n",
    "model = get_textrnn(config['word_maxlen']*2, embed_weights, pretrain=False)\n",
    "\n",
    "for i in range(15):\n",
    "    if i == 8:\n",
    "        K.set_value(model.optimizer.lr, 0.0001)\n",
    "\n",
    "    model.fit_generator(\n",
    "        train_batch_generator(x_train, y_train, config['batch_size']),\n",
    "        epochs=1,\n",
    "        steps_per_epoch=int(x_train.shape[0] / config['batch_size']),\n",
    "        validation_data=(x_dev, y_dev),\n",
    "    \n",
    "    )\n",
    "\n",
    "    pred = model.predict(x_dev, batch_size=config['batch_size'])\n",
    "    pre, rec, f1 = score(y_dev, pred)\n",
    "\n",
    "    print('p r f1 ', pre, rec, f1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:py2]",
   "language": "python",
   "name": "conda-env-py2-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
